

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.io.Writer;
import java.net.URL;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class CrawlerMain
{
	String folderName = "/media/Utils/UTA/CSE6339 Advance DB/Project/News Articles/crawled/";
	
	public static void main(String[] args)
	{
		CrawlerMain cm = new CrawlerMain();
		//cm.crawl();
		cm.crawlFTArticles();
    }

	private void crawlFTArticles()
	{
		//cnn archive
		String url1 = "http://www.floridatoday.com/section/CRIME";
		ArrayList<String> urls = new ArrayList<String>();
		urls.add(url1);
		for(String url : urls)
		{
			String retrievedText = downloadPage(url);
			if(!retrievedText.isEmpty()){
		        ArrayList<String> links = retrieveFTLinks(retrievedText);
		        int articleCnt = 1;
		        if(links != null)
		        {
			        for(String link : links)
			        {
			        	String newsContent =  downloadPage(link);
			        	String title =downloadCNNArticleTitle(newsContent);
			        	String content = downloadArticleBody(newsContent);
			        	
			        	break;
/*			        	if(!content.isEmpty()){
				        	String article=title +"\n"+content;
				        	//System.out.println(title +"\n"+content);
				        	String filename = folderName + title + articleCnt + ".txt";
				        	writeToFile(article, filename);
				        	System.out.println(articleCnt);
				        	articleCnt++;
			        	}
*/			        	
			        }
		        }
		        else
		        {
		        	System.out.println("ERROR: No links found in page: " + url);
		        }
			}
		}
	}

	private ArrayList<String> retrieveFTLinks(String fullText)
	{
	
		if(fullText.isEmpty())
			System.out.println("No text in the webpage.");
		ArrayList<String> links = new ArrayList<String>();
		//Pattern p = Pattern.compile("<li><a\\s+href\\s*=\\s*\"?(.*?)[\"|>]?(.*?)</a></li></ul>", Pattern.CASE_INSENSITIVE);
		// cnn archive
		Pattern p = Pattern.compile("<div class=\"content\">\\s*<div class=\"contentpic\">\\s*<a.*><img", Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(fullText);
		while(m.find())
		{
			String link = m.group().trim();
			//System.out.println(link);
			String hrefLink=null;
			if(link!=null){
				hrefLink = link.substring(link.indexOf("href=")+6,link.indexOf("\"><img"));
				hrefLink="http://www.floridatoday.com"+hrefLink+".html";
				links.add(hrefLink);
			}
			//System.out.println(hrefLink);
		}
		return links;
	
	}

	
	private void crawlCNNArticles()
	{
		//cnn archive
		String url1 = "http://www.cnn.com/CRIME/archive/index.html";
		ArrayList<String> urls = new ArrayList<String>();
		urls.add(url1);
		for(String url : urls)
		{
			String retrievedText = downloadPage(url);
			if(!retrievedText.isEmpty()){
		        ArrayList<String> links = retrieveCNNLinks(retrievedText);
		        int articleCnt = 1;
		        if(links != null)
		        {
			        for(String link : links)
			        {
			        	String newsContent =  downloadPage(link);
			        	String title =downloadCNNArticleTitle(newsContent);
			        	String content = downloadArticleBody(newsContent);
			        	if(!content.isEmpty()){
				        	String article=title +"\n"+content;
				        	//System.out.println(title +"\n"+content);
				        	String filename = folderName + title + articleCnt + ".txt";
				        	writeToFile(article, filename);
				        	System.out.println(articleCnt);
				        	articleCnt++;
			        	}
			        }
		        }
		        else
		        {
		        	System.out.println("ERROR: No links found in page: " + url);
		        }
			}
		}
	}
	
	private ArrayList<String> retrieveCNNLinks(String fullText)
	{
	
		if(fullText.isEmpty())
			System.out.println("No text in the webpage.");
		ArrayList<String> links = new ArrayList<String>();
		//Pattern p = Pattern.compile("<li><a\\s+href\\s*=\\s*\"?(.*?)[\"|>]?(.*?)</a></li></ul>", Pattern.CASE_INSENSITIVE);
		// cnn archive
		Pattern p = Pattern.compile("<div class=\"cnn_sclsts\">\\s*<a\\s+href\\s*=\\s*\"?(.*?)[\"|>]?(.*?)</a>\\s*</div>", Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(fullText);
		while(m.find())
		{
			String link = m.group().trim();
			String hrefLink=null;
			if(link!=null){
				hrefLink = link.substring(link.indexOf("http"),link.indexOf(".html")+5);
				links.add(hrefLink);
			}
			//System.out.println(link);
		}
		return links;
	
	}

	private String downloadArticleBody(String content)
	{
		String fullBody = "";
		try
		{
			//CNN Article
			//Pattern p = Pattern.compile("<p.*</p>", Pattern.MULTILINE);
			//FT
			Pattern p = Pattern.compile("<div id=\"artpagination\">\\s*.*</div>", Pattern.MULTILINE);
			Matcher m = p.matcher(content);
			while(m.find())
			{
				String data = m.group().trim();
				if(data!=null)
					fullBody+=data;
				System.out.println(fullBody);
			}
        }
        catch (Exception ex)
        {
            ex.printStackTrace();
        }
        return fullBody;
	}

	public String downloadCNNArticleTitle(String content){
		String title = "";
		try
		{
			Pattern p = Pattern.compile("<title>.*</title>", Pattern.MULTILINE);
			Matcher m = p.matcher(content);
			while(m.find())
			{
				String data = m.group().trim();
				if(data!=null)
					title=data.substring(data.indexOf("<title>")+7,data.indexOf("</title>") );
					System.out.println(title);
			}
        }
        catch (Exception ex)
        {
            ex.printStackTrace();
        }
        return title;
	}
	
	
	
	
	private String downloadPage(String url)
	{
		String fullText = "";
		try
		{
            URL my_url = new URL(url);
            BufferedReader br = new BufferedReader(new InputStreamReader(my_url.openStream()));
            String text = "";
            while(null != (text = br.readLine()))
            {
            	fullText += text + "\n";
            }
        }
        catch (Exception ex)
        {
            ex.printStackTrace();
        }
		System.out.println(fullText);
        return fullText;
        
	}
	

	
	private void crawl()
	{
		String url1 = "http://crimedaily.com/news/archives/2012_01.html";
		String url2 = "http://crimedaily.com/news/archives/2012_02.html";
		String url3 = "http://crimedaily.com/news/archives/2012_03.html";
		ArrayList<String> urls = new ArrayList<String>();
		urls.add(url1);
		urls.add(url2);
		urls.add(url3);
		int monthCnt = 1;
		for(String url : urls)
		{
			String retrievedText = downloadPage(url);
			if(retrievedText.isEmpty())
				continue;
	        ArrayList<String> links = retrieveLinks(retrievedText);
	        int articleCnt = 1;
	        if(links != null)
	        {
		        for(String link : links)
		        {
		        	String newsContent = downloadPage(link);
		        	if(newsContent.isEmpty())
		        		continue;
		        	String filename = folderName + monthCnt + "crimenews" + articleCnt + ".txt";
		        	//writeToFile(retrieveText(newsContent), filename);
		        	writeToFile(newsContent, filename);
		        	System.out.println(link);
		        	articleCnt++;
		        }
		        monthCnt++;
	        }
	        else
	        {
	        	System.out.println("ERROR: No links found in page: " + url);
	        }
		}
	}
	
	
	private ArrayList<String> retrieveLinks(String fullText)
	{
		if(fullText.isEmpty())
			System.out.println("No text in the webpage.");
		ArrayList<String> links = new ArrayList<String>();
		Pattern p = Pattern.compile("<B><a\\s+href\\s*=\\s*\"?(.*?)[\"|>]Read\\smore\\.\\.\\.", Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(fullText);
		while(m.find())
		{
			String link = m.group().trim();
			int start = link.indexOf("\"");
			int end = link.lastIndexOf("\"");
			links.add(link.substring(start+1, end));
			//System.out.println(link);
		}
		return links;
	}
	
	private String retrieveText(String fullText)
	{
		String newsContent = "";
		
		return newsContent;
	}
	
	private void writeToFile(String text, String outFilePath)
	{
		File outFile = new File(outFilePath);
		Writer out = null;
		try
		{
			out = new BufferedWriter(new FileWriter(outFile));
			out.write(text);
			out.close();
		}
		catch(Exception e)
		{
			e.printStackTrace();
		}		
	}
}